ML - A/B Testing
- tidyverse and tidyquant: These are the core data manipulation and visualization packages. We’ll mainly be using dplyr for data manipulation, ggplot2 for data visualization, and tidyquant themes for business reporting.
- parsnip, rsample, recipes, and yardstick: These are the tidyverse modeling packages. The parsnip package is an amazing tool that connects to the main machine learning algorithms. We teach parsnip in-depth (44 lessons, 5 hours of video) in Business Analysis with R, Week 6, Part 2 - Machine Learning (Regression).
- rpart, rpart.plot, and xgboost: These are the modeling libraries that we’ll connect to through the parsnip interface.
# Core packages
library(tidyverse)
library(tidyquant)
# Modeling packages
library(parsnip)
library(recipes)
library(rsample)
library(yardstick)
library(broom)
# Connector packages
library(rpart)
library(rpart.plot)
library(xgboost)
# Other packages
library(data.table)
library(plotly)
control_tbl <- fread("control_data.csv")
experiment_tbl <- fread("experiment_data.csv")
Investigate Data
head(control_tbl)
glimpse(control_tbl)
Rows: 37
Columns: 5
$ Date <chr> "Sat, Oct 11", "Sun, Oct 12", "Mon, Oct 13", "Tue, Oct 14", "Wed, Oct 15", "Thu, Oct 16", "Fri, Oct…
$ Pageviews <int> 7723, 9102, 10511, 9871, 10014, 9670, 9008, 7434, 8459, 10667, 10660, 9947, 8324, 9434, 8687, 8896,…
$ Clicks <int> 687, 779, 909, 836, 837, 823, 748, 632, 691, 861, 867, 838, 665, 673, 691, 708, 759, 736, 739, 734,…
$ Enrollments <int> 134, 147, 167, 156, 163, 138, 146, 110, 131, 165, 196, 162, 127, 220, 176, 161, 233, 154, 196, 167,…
$ Payments <int> 70, 70, 95, 105, 64, 82, 76, 70, 60, 97, 105, 92, 56, 122, 128, 104, 124, 91, 86, 75, 101, 93, 67, …
glimpse(experiment_tbl)
Rows: 37
Columns: 5
$ Date <chr> "Sat, Oct 11", "Sun, Oct 12", "Mon, Oct 13", "Tue, Oct 14", "Wed, Oct 15", "Thu, Oct 16", "Fri, Oct…
$ Pageviews <int> 7716, 9288, 10480, 9867, 9793, 9500, 9088, 7664, 8434, 10496, 10551, 9737, 8176, 9402, 8669, 8881, …
$ Clicks <int> 686, 785, 884, 827, 832, 788, 780, 652, 697, 860, 864, 801, 642, 697, 669, 693, 771, 736, 727, 728,…
$ Enrollments <int> 105, 116, 145, 138, 140, 129, 127, 94, 120, 153, 143, 128, 122, 194, 127, 153, 213, 162, 201, 207, …
$ Payments <int> 34, 91, 79, 92, 94, 61, 44, 62, 77, 98, 71, 70, 68, 94, 81, 101, 119, 120, 96, 67, 123, 100, 103, N…
Conclusions: - data is in character format; need to convert to date - payment is an outcome of enrollments so this should be removed
Check for Missing Data
control_tbl %>%
map_df(~ sum(is.na(.))) %>%
gather(key = "feature", value = "missing_count") %>%
arrange(desc(missing_count))
experiment_tbl %>%
map_df(~ sum(is.na(.))) %>%
gather(key = "feature", value = "missing_count") %>%
arrange(desc(missing_count))
control_tbl %>%
filter(is.na(Enrollments))
- we don’t have Enrollment information from November 3rd on. We will need to remove these observations
Format Data
- Combine the control_tbl and experiment_tbl, adding an “id” column indicating if the data was part of the experiment or not
- Add a “row_id” column to help for tracking which rows are selected for training and testing in the modeling section
- Create a “Day of Week” feature from the “Date” column
- Drop the unnecessary “Date” column and the “Payments” column
- Handle the missing data (NA) by removing these rows.
- Shuffle the rows to mix the data up for learning
- Reorganize the columns
set.seed(123)
data_formatted_tbl <- control_tbl %>%
# combine with experiment data
bind_rows(experiment_tbl, .id = "Experiment") %>%
mutate(Experiment = as.numeric(Experiment) - 1) %>%
# add row id
mutate(row_id = row_number()) %>%
# create a day of week feature
mutate(DOW = str_sub(Date, start = 1, end = 3) %>%
factor(levels = c("Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"))
) %>%
select(-Date, -Payments) %>%
# remove missing data
filter(!is.na(Enrollments)) %>%
# shuffle the data (note that set.seed is used to make reproducible)
slice_sample(prop = 1) %>%
# reorganize columns
select(row_id, Enrollments, Experiment, everything())
glimpse(data_formatted_tbl)
Rows: 46
Columns: 6
$ row_id <int> 45, 15, 14, 3, 56, 51, 58, 39, 40, 41, 5, 55, 42, 9, 43, 57, 8, 52, 7, 10, 47, 19, 4, 54, 17, 11, 4…
$ Enrollments <int> 94, 176, 220, 167, 201, 194, 182, 116, 145, 138, 163, 162, 140, 131, 129, 207, 110, 127, 146, 165, …
$ Experiment <dbl> 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, …
$ Pageviews <int> 7664, 8687, 9434, 10511, 9262, 9402, 8715, 9288, 10480, 9867, 10014, 9396, 9793, 8459, 9500, 9308, …
$ Clicks <int> 652, 691, 673, 909, 727, 697, 722, 785, 884, 827, 837, 736, 832, 691, 788, 728, 632, 669, 748, 861,…
$ DOW <fct> Sat, Sat, Fri, Mon, Wed, Fri, Fri, Sun, Mon, Tue, Wed, Tue, Wed, Sun, Thu, Thu, Sat, Sat, Fri, Mon,…
Split Data: Training and Testing
set.seed(123)
split_obj <- data_formatted_tbl %>% data.table() %>%
initial_split(prop = 0.8, strata = "Experiment")
train_tbl <- training(split_obj)
test_tbl <- testing(split_obj)
glimpse(train_tbl)
Rows: 36
Columns: 6
$ row_id <int> 15, 3, 5, 9, 8, 19, 4, 17, 11, 13, 20, 23, 18, 21, 6, 1, 2, 16, 45, 56, 51, 58, 39, 41, 55, 42, 43,…
$ Enrollments <int> 176, 167, 163, 131, 110, 196, 156, 233, 196, 127, 167, 206, 154, 174, 138, 134, 147, 161, 94, 201, …
$ Experiment <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, …
$ Pageviews <int> 8687, 10511, 10014, 8459, 7434, 9327, 9871, 9535, 10660, 8324, 9345, 8836, 9363, 8890, 9670, 7723, …
$ Clicks <int> 691, 909, 837, 691, 632, 739, 836, 759, 867, 665, 734, 693, 736, 706, 823, 687, 779, 708, 652, 727,…
$ DOW <fct> Sat, Mon, Wed, Sun, Sat, Wed, Tue, Mon, Tue, Thu, Thu, Sun, Tue, Fri, Thu, Sat, Sun, Sun, Sat, Wed,…
glimpse(test_tbl)
Rows: 10
Columns: 6
$ row_id <int> 14, 40, 52, 7, 10, 12, 59, 49, 22, 60
$ Enrollments <int> 220, 145, 127, 146, 165, 162, 142, 128, 156, 182
$ Experiment <dbl> 0, 1, 1, 0, 0, 0, 1, 1, 0, 1
$ Pageviews <int> 9434, 10480, 8669, 9008, 10667, 9947, 8448, 9737, 8460, 8836
$ Clicks <int> 673, 884, 669, 748, 861, 838, 695, 801, 681, 724
$ DOW <fct> Fri, Mon, Sat, Fri, Mon, Wed, Sat, Wed, Sat, Sun
Implement ML Algorithms
- implement 3 modeling approaches:
- Linear Regression - Linear, Explainable (Baseline)
- Decision Tree - Pros: Non-Linear, Explainable; Cons: Lower Performance
- XGBoost - Pros: Non-Linear, High Performance; Cons: Less Explainable
Linear Regression (Baseline)
- create a model using the training data and predict on the test data
model_01_lm <- linear_reg("regression") %>%
set_engine("lm") %>%
fit(Enrollments ~ ., data = train_tbl %>% select(-row_id))
pred_01_lm <- model_01_lm %>%
predict(new_data = test_tbl) %>%
bind_cols(test_tbl %>% select(Enrollments))
pred_01_lm %>%
metrics(truth = Enrollments, estimate = .pred) %>%
knitr::kable()
| rmse |
standard |
32.1813991 |
| rsq |
standard |
0.3444336 |
| mae |
standard |
27.7148176 |
pred_01_lm %>%
mutate(observation = row_number() %>% as.character()) %>%
gather(key = "key", value = "value", -observation, factor_key = TRUE) %>%
plot_ly(x = ~observation, y = ~value, color = ~key, type = "scatter", mode = "markers") %>%
layout(title = "Prediction vs Annual Enrollments: Model 1 - Linear Regression")
What’s driving this model?
Use the tidy() function from the broom package to help. This gets us the model estimates. We can arrange by “p.value” to get an idea of how important the model terms are. Clicks, Pageviews, and Experiment are judged strong predictors with a p-value less than 0.05. However, we want to try out other modeling techniques to judge this. We note that the coefficient of Experiment is -17.6, and because the term is binary (0 or 1) this can be interpreted as decreasing Enrollments by -17.6 per day when the Experiment is run.
linear_regression_model_terms_tbl <- model_01_lm$fit %>%
tidy() %>%
arrange(p.value) %>%
mutate(term = as.factor(term) %>% fct_rev())
linear_regression_model_terms_tbl %>% knitr::kable()
| Clicks |
-0.6709381 |
0.1351726 |
-4.9635647 |
0.0000370 |
| Pageviews |
0.0692229 |
0.0154320 |
4.4856713 |
0.0001306 |
| Experiment |
-16.8809060 |
7.3964356 |
-2.2823029 |
0.0308955 |
| DOWMon |
31.1476094 |
17.1397658 |
1.8172716 |
0.0807195 |
| DOWFri |
18.0185100 |
14.0265834 |
1.2845972 |
0.2102610 |
| DOWWed |
18.3067550 |
15.1370069 |
1.2094039 |
0.2373904 |
| DOWSat |
10.2618805 |
16.4042902 |
0.6255608 |
0.5370559 |
| DOWThu |
-6.6848206 |
12.3334080 |
-0.5420092 |
0.5924275 |
| (Intercept) |
26.5108797 |
77.5271192 |
0.3419562 |
0.7351325 |
| DOWTue |
-4.6330479 |
15.4975121 |
-0.2989543 |
0.7673515 |
Visualizing the features show that clicks, pageviews and experiment are the most important features and are likely to be the best predictors
linear_regression_model_terms_tbl %>%
arrange(desc(p.value), term) %>%
plot_ly(x = ~p.value, y = ~term, type = "scatter", mode = "markers") %>%
add_lines(x = ~0.05) %>%
layout(title = "Feature Importance: Model 1 - Linear Regression")
Set-up Helper Functions
calc_metrics <- function(model, new_data) {
model %>%
predict(new_data = new_data) %>%
bind_cols(new_data %>% select(Enrollments)) %>%
metrics(truth = Enrollments,
estimate = .pred)
}
plot_predictions <- function(model, new_data) {
predict(model, new_data) %>%
bind_cols(new_data %>% select(Enrollments)) %>%
mutate(observation = row_number() %>% as.character()) %>%
gather(key = "key", value = "value", -observation, factor_key = TRUE) %>%
plot_ly(x = ~observation, y = ~value, color = ~key, type = "scatter", mode = "markers")
}
Decision Trees
Decision Trees are excellent models that can pick up on non-linearities and often make very informative models that compliment linear models by providing a different way of viewing the problem. We can implement a decision tree with decision_tree(). We’ll set the engine to “rpart”, a popular decision tree package. There are a few key tunable parameters: - cost_complexity: A cutoff for model splitting based on increase in explainability - tree_depth: The max tree depth - min_n: The minimum number of observations in terminal (leaf) nodes The parameters selected for the model were determined using 5-fold cross validation to prevent over-fitting.
model_02_decision_tree <- decision_tree(
mode = "regression",
cost_complexity = 0.001,
tree_depth = 5,
min_n = 4) %>%
set_engine("rpart") %>%
fit(Enrollments ~ ., data = train_tbl %>% select(-row_id))
Next, calculate the metrics on this model using our helper function, calc_metrics(). The MAE of the predictions is approximately the same as the linear model at +/-19 Enrollments per day.
model_02_decision_tree %>%
calc_metrics(test_tbl) %>%
knitr::kable()
| rmse |
standard |
22.71472 |
| rsq |
standard |
0.38662 |
| mae |
standard |
18.08333 |
model_02_decision_tree %>%
plot_predictions(test_tbl) %>%
layout(title = "Prediction vs Actual Enrollments: Decision Tree")
Warning in RColorBrewer::brewer.pal(N, "Set2") :
minimal value for n is 3, returning requested palette with 3 different levels
Warning in RColorBrewer::brewer.pal(N, "Set2") :
minimal value for n is 3, returning requested palette with 3 different levels
Warning in RColorBrewer::brewer.pal(N, "Set2") :
minimal value for n is 3, returning requested palette with 3 different levels
Warning in RColorBrewer::brewer.pal(N, "Set2") :
minimal value for n is 3, returning requested palette with 3 different levels
Finally, use rpart.plot() to visualize the decision tree rules. Note that we need to extract the underlying “rpart” model from the parsnip model object using the model_02_decision_tree$fit
model_02_decision_tree$fit %>%
rpart.plot(roundint = FALSE,
cex = 0.8,
fallen.leaves = TRUE,
extra = 101,
main = "Model 02: Decision Tree")

Interpreting the decision tree is straightforward: Each decision is a rule, and Yes is to the left, No is to the right. The top features are the most important to the model (“Pageviews” and “Clicks”). The decision tree shows that “Experiment” is involved in the decision rules. The rules indicate a when Experiment >= 0.5, there is a drop in enrollments.
Key Points:
Our new model has roughly the same accuracy to +/-19 enrollments (MAE) as the linear regression model.
Experiment shows up towards the bottom of the tree. The rules indicate a when Experiment >= 0.5, there is a drop in enrollments.
XGBoost
Several key tuning parameters include: - mtry: The number of predictors that will be randomly sampled at each split when creating the tree models. - trees: The number of trees contained in the ensemble. - min_n: The minimum number of data points in a node that are required for the node to be split further. - tree_depth: The maximum depth of the tree (i.e. number of splits). - learn_rate: The rate at which the boosting algorithm adapts from iteration-to-iteration. - loss_reduction: The reduction in the loss function required to split further. - sample_size: The amount of data exposed to the fitting routine.
The parameters selected for the model were determined using 5-fold cross validation to prevent over-fitting.
set.seed(123)
model_03_xgboost <- boost_tree(
mode = "regression",
mtry = 100,
trees = 1000,
min_n = 8,
tree_depth = 6,
learn_rate = 0.2,
loss_reduction = 0.01,
sample_size = 1) %>%
set_engine("xgboost") %>%
fit(Enrollments ~ ., data = train_tbl %>% select(-row_id))
model_03_xgboost %>%
calc_metrics(test_tbl) %>%
knitr::kable()
| rmse |
standard |
20.5911658 |
| rsq |
standard |
0.4683664 |
| mae |
standard |
17.5331612 |
model_03_xgboost %>%
plot_predictions(test_tbl) %>%
layout(title = "Prediction vs Actual Enrollments: XGBoost")
Warning in RColorBrewer::brewer.pal(N, "Set2") :
minimal value for n is 3, returning requested palette with 3 different levels
Warning in RColorBrewer::brewer.pal(N, "Set2") :
minimal value for n is 3, returning requested palette with 3 different levels
Warning in RColorBrewer::brewer.pal(N, "Set2") :
minimal value for n is 3, returning requested palette with 3 different levels
Warning in RColorBrewer::brewer.pal(N, "Set2") :
minimal value for n is 3, returning requested palette with 3 different levels
Feature Importance
xgboost_feature_importance_tbl <- model_03_xgboost$fit %>%
xgb.importance(model = .) %>%
as_tibble() %>%
mutate(Feature = as_factor(Feature) %>% fct_rev())
xgboost_feature_importance_tbl %>% knitr::kable()
| Pageviews |
0.5551216 |
0.5979430 |
0.5762712 |
| Clicks |
0.3824186 |
0.2145172 |
0.2364613 |
| Experiment |
0.0624599 |
0.1875398 |
0.1872675 |
xgboost_feature_importance_tbl %>%
mutate(Label = paste0(round(Gain*100, 1)), "%") %>%
plot_ly(x = ~Gain, y = ~Feature, type = "scatter", mode = "markers", name = ~Label) %>%
layout(title = "XGBoost Feature Importance")
The information gain is 93% from Pageviews and Clicks combined. Experiment has about a 7% contribution to information gain, indicating it’s still predictive (just not nearly as much as Pageviews). This tells a story that if Enrollments are critical, Udacity should focus on getting Pageviews.
Key Points: - The XGBoost model error has dropped to +/-11 Enrollments. - The XGBoost shows that Experiment provides an information gain of 7% - The XGBoost model tells a story that Udacity should be focusing on Page Views and secondarily Clicks to maintain or increase Enrollments. The features drive the system.
Business Conclusions
There are several key benefits to performing A/B Testing using Machine Learning. These include:
Understanding the Complex System - We discovered that the system is driven by Pageviews and Clicks. Statistical Inference would not have identified these drivers. Machine Learning did.
Providing a direction and magnitude of the experiment - We saw that Experiment = 1 drops enrollments by -17.6 Enrollments Per Day in the Linear Regression. We saw similar drops in the Decision Tree rules. Statistical inference would not have identified magnitude and direction. Only whether or not the Experiment had an effect.
What Should Udacity Do?
If Udacity wants to maximimize enrollments, it should focus on increasing Page Views from qualified candidates. Page Views is the most important feature in 2 of 3 models.
If Udacity wants alert people of the time commitment, the additional popup form is expected to decrease the number of enrollments. The negative impact can be seen in the decision tree (when Experiment <= 0.5, Enrollments go down) and in the linear regression model term (-17.6 Enrollments when Experiment = 1). Is this OK? It depends on what Udacity’s goals are.
Using caret
LS0tCnRpdGxlOiAiQS9CIFRlc3RpbmciCm91dHB1dDogaHRtbF9ub3RlYm9vawotLS0KCiMgVHV0b3JpYWwgU291cmNlOgpodHRwczovL3d3dy5idXNpbmVzcy1zY2llbmNlLmlvL2J1c2luZXNzLzIwMTkvMDMvMTEvYWItdGVzdGluZy1tYWNoaW5lLWxlYXJuaW5nLmh0bWwKCiMgQS9CIFRlc3RpbmcKLSBha2Egc3BsaXQgdGVzdGluZyBvciBidWNrZXQgdGVzdGluZwotIGluIG1hcmtldGluZywgQS9CIHRlc3RpbmcgZW5hYmxlcyB1cyB0byBkZXRlcm1pbmUgd2hldGhlciBjaGFuZ2VzIGluIGxhbmRpbmcgcGFnZXMsIHBvcHVwIGZvcm1zLCBhcnRpY2xlIHRpdGxlcywgYW5kIG90aGVyIGRpZ2l0YWwgbWFya2V0aW5nIGRlY2lzaW9ucyBpbXByb3ZlIGNvbnZlcnNpb24gcmF0ZXMgYW5kIHVsdGltYXRlbHkgY3VzdG9tZXIgcHVyY2hhc2luZyBmYXZvcgotIGEgc3VjY2Vzc2Z1bCBBL0IgdGVzdGluZyBzdHJhdGVneSBjYW4gbGVhZCB0byBtYXNzaXZlIGdhaW5zIC0gbW9yZSBzYXRpc2ZpZWQgdXNlcnMsIG1vcmUgZW5nYWdlbWVudCwgYW5kIG1vcmUgc2FsZXMKCiMjIE1ldGhvZAoyIHRlc3RzIGFyZSBydW4gaW4gcGFyYWxsZWw6CjEuIFRyZWF0bWVudCBHcm91cCAoR3JvdXAgQSkgLSBncm91cCBleHBvc2VkIHRvIHRoZSBuZXcgd2ViIHBhZ2UsIHBvcHVwIGZvcm0sIGV0YwoyLiBDb250cm9sIEdyb3VwIChHcm91cCBCKSAtIGdyb3VwIGV4cGVyaWVuY2VzIG5vIGNoYW5nZSBmcm9tIHRoZSBjdXJyZW50IHNldHVwCi0gZ29hbCBpcyB0byBjb21wYXJlIHRoZSBjb252ZXJzaW9uIHJhdGVzIG9mIHRoZSB0d28gZ3JvdXBzIHVzaW5nIHN0YXRpc3RpY2FsIGluZmVyZW5jZQoKIyBNTCAtIEEvQiBUZXN0aW5nIAotIHRpZHl2ZXJzZSBhbmQgdGlkeXF1YW50OiBUaGVzZSBhcmUgdGhlIGNvcmUgZGF0YSBtYW5pcHVsYXRpb24gYW5kIHZpc3VhbGl6YXRpb24gcGFja2FnZXMuIFdl4oCZbGwgbWFpbmx5IGJlIHVzaW5nIGRwbHlyIGZvciBkYXRhIG1hbmlwdWxhdGlvbiwgZ2dwbG90MiBmb3IgZGF0YSB2aXN1YWxpemF0aW9uLCBhbmQgdGlkeXF1YW50IHRoZW1lcyBmb3IgYnVzaW5lc3MgcmVwb3J0aW5nLgotIHBhcnNuaXAsIHJzYW1wbGUsIHJlY2lwZXMsIGFuZCB5YXJkc3RpY2s6IFRoZXNlIGFyZSB0aGUgdGlkeXZlcnNlIG1vZGVsaW5nIHBhY2thZ2VzLiBUaGUgcGFyc25pcCBwYWNrYWdlIGlzIGFuIGFtYXppbmcgdG9vbCB0aGF0IGNvbm5lY3RzIHRvIHRoZSBtYWluIG1hY2hpbmUgbGVhcm5pbmcgYWxnb3JpdGhtcy4gV2UgdGVhY2ggcGFyc25pcCBpbi1kZXB0aCAoNDQgbGVzc29ucywgNSBob3VycyBvZiB2aWRlbykgaW4gQnVzaW5lc3MgQW5hbHlzaXMgd2l0aCBSLCBXZWVrIDYsIFBhcnQgMiAtIE1hY2hpbmUgTGVhcm5pbmcgKFJlZ3Jlc3Npb24pLgotIHJwYXJ0LCBycGFydC5wbG90LCBhbmQgeGdib29zdDogVGhlc2UgYXJlIHRoZSBtb2RlbGluZyBsaWJyYXJpZXMgdGhhdCB3ZeKAmWxsIGNvbm5lY3QgdG8gdGhyb3VnaCB0aGUgcGFyc25pcCBpbnRlcmZhY2UuCgpgYGB7ciBsaWJyYXJpZXMsIHdhcm5pbmc9RkFMU0V9CiMgQ29yZSBwYWNrYWdlcwpsaWJyYXJ5KHRpZHl2ZXJzZSkKbGlicmFyeSh0aWR5cXVhbnQpCgojIE1vZGVsaW5nIHBhY2thZ2VzCmxpYnJhcnkocGFyc25pcCkKbGlicmFyeShyZWNpcGVzKQpsaWJyYXJ5KHJzYW1wbGUpCmxpYnJhcnkoeWFyZHN0aWNrKQpsaWJyYXJ5KGJyb29tKQoKIyBDb25uZWN0b3IgcGFja2FnZXMKbGlicmFyeShycGFydCkKbGlicmFyeShycGFydC5wbG90KQpsaWJyYXJ5KHhnYm9vc3QpCgojIE90aGVyIHBhY2thZ2VzCmxpYnJhcnkoZGF0YS50YWJsZSkKbGlicmFyeShwbG90bHkpCmBgYAoKYGBge3IgbG9hZF9kYXRhfQpjb250cm9sX3RibCA8LSBmcmVhZCgiY29udHJvbF9kYXRhLmNzdiIpCmV4cGVyaW1lbnRfdGJsIDwtIGZyZWFkKCJleHBlcmltZW50X2RhdGEuY3N2IikKYGBgCgojIyBJbnZlc3RpZ2F0ZSBEYXRhCgpgYGB7cn0KaGVhZChjb250cm9sX3RibCkgCmBgYAoKYGBge3J9CmdsaW1wc2UoY29udHJvbF90YmwpCmBgYAoKYGBge3J9CmdsaW1wc2UoZXhwZXJpbWVudF90YmwpCmBgYApDb25jbHVzaW9uczoKLSBkYXRhIGlzIGluIGNoYXJhY3RlciBmb3JtYXQ7IG5lZWQgdG8gY29udmVydCB0byBkYXRlCi0gcGF5bWVudCBpcyBhbiBvdXRjb21lIG9mIGVucm9sbG1lbnRzIHNvIHRoaXMgc2hvdWxkIGJlIHJlbW92ZWQKCiMjIENoZWNrIGZvciBNaXNzaW5nIERhdGEKYGBge3J9CmNvbnRyb2xfdGJsICU+JSAKICBtYXBfZGYofiBzdW0oaXMubmEoLikpKSAlPiUgCiAgZ2F0aGVyKGtleSA9ICJmZWF0dXJlIiwgdmFsdWUgPSAibWlzc2luZ19jb3VudCIpICU+JSAKICBhcnJhbmdlKGRlc2MobWlzc2luZ19jb3VudCkpCmBgYAoKYGBge3J9CmV4cGVyaW1lbnRfdGJsICU+JSAKICBtYXBfZGYofiBzdW0oaXMubmEoLikpKSAlPiUgCiAgZ2F0aGVyKGtleSA9ICJmZWF0dXJlIiwgdmFsdWUgPSAibWlzc2luZ19jb3VudCIpICU+JSAKICBhcnJhbmdlKGRlc2MobWlzc2luZ19jb3VudCkpCmBgYAoKYGBge3J9CmNvbnRyb2xfdGJsICU+JSAKICBmaWx0ZXIoaXMubmEoRW5yb2xsbWVudHMpKQpgYGAKLSB3ZSBkb24ndCBoYXZlIEVucm9sbG1lbnQgaW5mb3JtYXRpb24gZnJvbSBOb3ZlbWJlciAzcmQgb24uICBXZSB3aWxsIG5lZWQgdG8gcmVtb3ZlIHRoZXNlIG9ic2VydmF0aW9ucwoKIyMgRm9ybWF0IERhdGEKLSBDb21iaW5lIHRoZSBjb250cm9sX3RibCBhbmQgZXhwZXJpbWVudF90YmwsIGFkZGluZyBhbiDigJxpZOKAnSBjb2x1bW4gaW5kaWNhdGluZyBpZiB0aGUgZGF0YSB3YXMgcGFydCBvZiB0aGUgZXhwZXJpbWVudCBvciBub3QKLSBBZGQgYSDigJxyb3dfaWTigJ0gY29sdW1uIHRvIGhlbHAgZm9yIHRyYWNraW5nIHdoaWNoIHJvd3MgYXJlIHNlbGVjdGVkIGZvciB0cmFpbmluZyBhbmQgdGVzdGluZyBpbiB0aGUgbW9kZWxpbmcgc2VjdGlvbgotIENyZWF0ZSBhIOKAnERheSBvZiBXZWVr4oCdIGZlYXR1cmUgZnJvbSB0aGUg4oCcRGF0ZeKAnSBjb2x1bW4KLSBEcm9wIHRoZSB1bm5lY2Vzc2FyeSDigJxEYXRl4oCdIGNvbHVtbiBhbmQgdGhlIOKAnFBheW1lbnRz4oCdIGNvbHVtbgotIEhhbmRsZSB0aGUgbWlzc2luZyBkYXRhIChOQSkgYnkgcmVtb3ZpbmcgdGhlc2Ugcm93cy4KLSBTaHVmZmxlIHRoZSByb3dzIHRvIG1peCB0aGUgZGF0YSB1cCBmb3IgbGVhcm5pbmcKLSBSZW9yZ2FuaXplIHRoZSBjb2x1bW5zCgpgYGB7cn0Kc2V0LnNlZWQoMTIzKQoKZGF0YV9mb3JtYXR0ZWRfdGJsIDwtIGNvbnRyb2xfdGJsICU+JSAKICAKICAjIGNvbWJpbmUgd2l0aCBleHBlcmltZW50IGRhdGEKICBiaW5kX3Jvd3MoZXhwZXJpbWVudF90YmwsIC5pZCA9ICJFeHBlcmltZW50IikgJT4lIAogIG11dGF0ZShFeHBlcmltZW50ID0gYXMubnVtZXJpYyhFeHBlcmltZW50KSAtIDEpICU+JSAKICAKICAjIGFkZCByb3cgaWQKICBtdXRhdGUocm93X2lkID0gcm93X251bWJlcigpKSAlPiUgCiAgCiAgIyBjcmVhdGUgYSBkYXkgb2Ygd2VlayBmZWF0dXJlCiAgbXV0YXRlKERPVyA9IHN0cl9zdWIoRGF0ZSwgc3RhcnQgPSAxLCBlbmQgPSAzKSAlPiUgCiAgICAgICAgICAgZmFjdG9yKGxldmVscyA9IGMoIlN1biIsICJNb24iLCAiVHVlIiwgIldlZCIsICJUaHUiLCAiRnJpIiwgIlNhdCIpKQogICAgICAgICApICU+JSAKICBzZWxlY3QoLURhdGUsIC1QYXltZW50cykgJT4lIAogIAogICMgcmVtb3ZlIG1pc3NpbmcgZGF0YQogIGZpbHRlcighaXMubmEoRW5yb2xsbWVudHMpKSAlPiUgCiAgCiAgIyBzaHVmZmxlIHRoZSBkYXRhIChub3RlIHRoYXQgc2V0LnNlZWQgaXMgdXNlZCB0byBtYWtlIHJlcHJvZHVjaWJsZSkKICBzbGljZV9zYW1wbGUocHJvcCA9IDEpICU+JSAKICAKICAjIHJlb3JnYW5pemUgY29sdW1ucwogIHNlbGVjdChyb3dfaWQsIEVucm9sbG1lbnRzLCBFeHBlcmltZW50LCBldmVyeXRoaW5nKCkpCgpnbGltcHNlKGRhdGFfZm9ybWF0dGVkX3RibCkKYGBgCiMjIFNwbGl0IERhdGE6IFRyYWluaW5nIGFuZCBUZXN0aW5nCgpgYGB7cn0Kc2V0LnNlZWQoMTIzKQoKc3BsaXRfb2JqIDwtIGRhdGFfZm9ybWF0dGVkX3RibCAlPiUgZGF0YS50YWJsZSgpICU+JSAKICBpbml0aWFsX3NwbGl0KHByb3AgPSAwLjgsIHN0cmF0YSA9ICJFeHBlcmltZW50IikKCnRyYWluX3RibCA8LSB0cmFpbmluZyhzcGxpdF9vYmopCnRlc3RfdGJsIDwtIHRlc3Rpbmcoc3BsaXRfb2JqKQpgYGAKCmBgYHtyfQpnbGltcHNlKHRyYWluX3RibCkKYGBgCgpgYGB7cn0KZ2xpbXBzZSh0ZXN0X3RibCkKYGBgCiMjIEltcGxlbWVudCBNTCBBbGdvcml0aG1zCi0gaW1wbGVtZW50IDMgbW9kZWxpbmcgYXBwcm9hY2hlczoKMS4gTGluZWFyIFJlZ3Jlc3Npb24gLSBMaW5lYXIsIEV4cGxhaW5hYmxlIChCYXNlbGluZSkKMi4gRGVjaXNpb24gVHJlZSAtIFByb3M6IE5vbi1MaW5lYXIsIEV4cGxhaW5hYmxlOyBDb25zOiBMb3dlciBQZXJmb3JtYW5jZQozLiBYR0Jvb3N0IC0gUHJvczogTm9uLUxpbmVhciwgSGlnaCBQZXJmb3JtYW5jZTsgQ29uczogTGVzcyBFeHBsYWluYWJsZQoKIyMjIExpbmVhciBSZWdyZXNzaW9uIChCYXNlbGluZSkKLSBjcmVhdGUgYSBtb2RlbCB1c2luZyB0aGUgdHJhaW5pbmcgZGF0YSBhbmQgcHJlZGljdCBvbiB0aGUgdGVzdCBkYXRhIApgYGB7cn0KbW9kZWxfMDFfbG0gPC0gbGluZWFyX3JlZygicmVncmVzc2lvbiIpICU+JSAKICBzZXRfZW5naW5lKCJsbSIpICU+JSAKICBmaXQoRW5yb2xsbWVudHMgfiAuLCBkYXRhID0gdHJhaW5fdGJsICU+JSBzZWxlY3QoLXJvd19pZCkpCmBgYAoKYGBge3J9CnByZWRfMDFfbG0gPC0gbW9kZWxfMDFfbG0gJT4lCiAgcHJlZGljdChuZXdfZGF0YSA9IHRlc3RfdGJsKSAlPiUKICBiaW5kX2NvbHModGVzdF90YmwgJT4lIHNlbGVjdChFbnJvbGxtZW50cykpIAoKcHJlZF8wMV9sbSAlPiUgCiAgbWV0cmljcyh0cnV0aCA9IEVucm9sbG1lbnRzLCBlc3RpbWF0ZSA9IC5wcmVkKSAlPiUgCiAga25pdHI6OmthYmxlKCkKYGBgCgpgYGB7ciB3YXJuaW5nPUZBTFNFLCBmaWcud2lkdGg9Nn0KcHJlZF8wMV9sbSAlPiUgCiAgbXV0YXRlKG9ic2VydmF0aW9uID0gcm93X251bWJlcigpICU+JSBhcy5jaGFyYWN0ZXIoKSkgJT4lIAogIGdhdGhlcihrZXkgPSAia2V5IiwgdmFsdWUgPSAidmFsdWUiLCAtb2JzZXJ2YXRpb24sIGZhY3Rvcl9rZXkgPSBUUlVFKSAlPiUgCiAgcGxvdF9seSh4ID0gfm9ic2VydmF0aW9uLCB5ID0gfnZhbHVlLCBjb2xvciA9IH5rZXksIHR5cGUgPSAic2NhdHRlciIsIG1vZGUgPSAibWFya2VycyIpICU+JQogIGxheW91dCh0aXRsZSA9ICJQcmVkaWN0aW9uIHZzIEFubnVhbCBFbnJvbGxtZW50czogTW9kZWwgMSAtIExpbmVhciBSZWdyZXNzaW9uIikKYGBgCiMjIyBXaGF0J3MgZHJpdmluZyB0aGlzIG1vZGVsPwpVc2UgdGhlIHRpZHkoKSBmdW5jdGlvbiBmcm9tIHRoZSBicm9vbSBwYWNrYWdlIHRvIGhlbHAuIFRoaXMgZ2V0cyB1cyB0aGUgbW9kZWwgZXN0aW1hdGVzLiBXZSBjYW4gYXJyYW5nZSBieSDigJxwLnZhbHVl4oCdIHRvIGdldCBhbiBpZGVhIG9mIGhvdyBpbXBvcnRhbnQgdGhlIG1vZGVsIHRlcm1zIGFyZS4gQ2xpY2tzLCBQYWdldmlld3MsIGFuZCBFeHBlcmltZW50IGFyZSBqdWRnZWQgc3Ryb25nIHByZWRpY3RvcnMgd2l0aCBhIHAtdmFsdWUgbGVzcyB0aGFuIDAuMDUuIEhvd2V2ZXIsIHdlIHdhbnQgdG8gdHJ5IG91dCBvdGhlciBtb2RlbGluZyB0ZWNobmlxdWVzIHRvIGp1ZGdlIHRoaXMuIFdlIG5vdGUgdGhhdCB0aGUgY29lZmZpY2llbnQgb2YgRXhwZXJpbWVudCBpcyAtMTcuNiwgYW5kIGJlY2F1c2UgdGhlIHRlcm0gaXMgYmluYXJ5ICgwIG9yIDEpIHRoaXMgY2FuIGJlIGludGVycHJldGVkIGFzIGRlY3JlYXNpbmcgRW5yb2xsbWVudHMgYnkgLTE3LjYgcGVyIGRheSB3aGVuIHRoZSBFeHBlcmltZW50IGlzIHJ1bi4KCmBgYHtyfQpsaW5lYXJfcmVncmVzc2lvbl9tb2RlbF90ZXJtc190YmwgPC0gbW9kZWxfMDFfbG0kZml0ICU+JSAKICB0aWR5KCkgJT4lIAogIGFycmFuZ2UocC52YWx1ZSkgJT4lIAogIG11dGF0ZSh0ZXJtID0gYXMuZmFjdG9yKHRlcm0pICU+JSBmY3RfcmV2KCkpCgpsaW5lYXJfcmVncmVzc2lvbl9tb2RlbF90ZXJtc190YmwgJT4lIGtuaXRyOjprYWJsZSgpCmBgYApWaXN1YWxpemluZyB0aGUgZmVhdHVyZXMgc2hvdyB0aGF0IGNsaWNrcywgcGFnZXZpZXdzIGFuZCBleHBlcmltZW50IGFyZSB0aGUgbW9zdCBpbXBvcnRhbnQgZmVhdHVyZXMgYW5kIGFyZSBsaWtlbHkgdG8gYmUgdGhlIGJlc3QgcHJlZGljdG9ycyAKYGBge3J9CmxpbmVhcl9yZWdyZXNzaW9uX21vZGVsX3Rlcm1zX3RibCAlPiUgCiAgYXJyYW5nZShkZXNjKHAudmFsdWUpLCB0ZXJtKSAlPiUgCiAgcGxvdF9seSh4ID0gfnAudmFsdWUsIHkgPSB+dGVybSwgdHlwZSA9ICJzY2F0dGVyIiwgbW9kZSA9ICJtYXJrZXJzIikgJT4lIAogIGFkZF9saW5lcyh4ID0gfjAuMDUpICU+JSAKICBsYXlvdXQodGl0bGUgPSAiRmVhdHVyZSBJbXBvcnRhbmNlOiBNb2RlbCAxIC0gTGluZWFyIFJlZ3Jlc3Npb24iKQpgYGAKIyMgU2V0LXVwIEhlbHBlciBGdW5jdGlvbnMKCmBgYHtyfQpjYWxjX21ldHJpY3MgPC0gZnVuY3Rpb24obW9kZWwsIG5ld19kYXRhKSB7CiAgbW9kZWwgJT4lCiAgICBwcmVkaWN0KG5ld19kYXRhID0gbmV3X2RhdGEpICU+JSAKICAgIGJpbmRfY29scyhuZXdfZGF0YSAlPiUgc2VsZWN0KEVucm9sbG1lbnRzKSkgJT4lIAogICAgbWV0cmljcyh0cnV0aCA9IEVucm9sbG1lbnRzLCAKICAgICAgICAgICAgZXN0aW1hdGUgPSAucHJlZCkKfQpgYGAKCmBgYHtyfQpwbG90X3ByZWRpY3Rpb25zIDwtIGZ1bmN0aW9uKG1vZGVsLCBuZXdfZGF0YSkgewogIHByZWRpY3QobW9kZWwsIG5ld19kYXRhKSAlPiUgCiAgICBiaW5kX2NvbHMobmV3X2RhdGEgJT4lIHNlbGVjdChFbnJvbGxtZW50cykpICU+JSAKICAgIG11dGF0ZShvYnNlcnZhdGlvbiA9IHJvd19udW1iZXIoKSAlPiUgYXMuY2hhcmFjdGVyKCkpICU+JSAKICAgIGdhdGhlcihrZXkgPSAia2V5IiwgdmFsdWUgPSAidmFsdWUiLCAtb2JzZXJ2YXRpb24sIGZhY3Rvcl9rZXkgPSBUUlVFKSAlPiUgCiAgICBwbG90X2x5KHggPSB+b2JzZXJ2YXRpb24sIHkgPSB+dmFsdWUsIGNvbG9yID0gfmtleSwgdHlwZSA9ICJzY2F0dGVyIiwgbW9kZSA9ICJtYXJrZXJzIikKfQpgYGAKCiMjIERlY2lzaW9uIFRyZWVzCkRlY2lzaW9uIFRyZWVzIGFyZSBleGNlbGxlbnQgbW9kZWxzIHRoYXQgY2FuIHBpY2sgdXAgb24gbm9uLWxpbmVhcml0aWVzIGFuZCBvZnRlbiBtYWtlIHZlcnkgaW5mb3JtYXRpdmUgbW9kZWxzIHRoYXQgY29tcGxpbWVudCBsaW5lYXIgbW9kZWxzIGJ5IHByb3ZpZGluZyBhIGRpZmZlcmVudCB3YXkgb2Ygdmlld2luZyB0aGUgcHJvYmxlbS4KV2UgY2FuIGltcGxlbWVudCBhIGRlY2lzaW9uIHRyZWUgd2l0aCBkZWNpc2lvbl90cmVlKCkuIFdl4oCZbGwgc2V0IHRoZSBlbmdpbmUgdG8g4oCccnBhcnTigJ0sIGEgcG9wdWxhciBkZWNpc2lvbiB0cmVlIHBhY2thZ2UuIFRoZXJlIGFyZSBhIGZldyBrZXkgdHVuYWJsZSBwYXJhbWV0ZXJzOgotIGNvc3RfY29tcGxleGl0eTogQSBjdXRvZmYgZm9yIG1vZGVsIHNwbGl0dGluZyBiYXNlZCBvbiBpbmNyZWFzZSBpbiBleHBsYWluYWJpbGl0eQotIHRyZWVfZGVwdGg6IFRoZSBtYXggdHJlZSBkZXB0aAotIG1pbl9uOiBUaGUgbWluaW11bSBudW1iZXIgb2Ygb2JzZXJ2YXRpb25zIGluIHRlcm1pbmFsIChsZWFmKSBub2RlcwpUaGUgcGFyYW1ldGVycyBzZWxlY3RlZCBmb3IgdGhlIG1vZGVsIHdlcmUgZGV0ZXJtaW5lZCB1c2luZyA1LWZvbGQgY3Jvc3MgdmFsaWRhdGlvbiB0byBwcmV2ZW50IG92ZXItZml0dGluZy4KCmBgYHtyfQptb2RlbF8wMl9kZWNpc2lvbl90cmVlIDwtIGRlY2lzaW9uX3RyZWUoCiAgICBtb2RlID0gInJlZ3Jlc3Npb24iLCAKICAgIGNvc3RfY29tcGxleGl0eSA9IDAuMDAxLCAKICAgIHRyZWVfZGVwdGggPSA1LCAKICAgIG1pbl9uID0gNCkgJT4lIAogIHNldF9lbmdpbmUoInJwYXJ0IikgJT4lIAogIGZpdChFbnJvbGxtZW50cyB+IC4sIGRhdGEgPSB0cmFpbl90YmwgJT4lIHNlbGVjdCgtcm93X2lkKSkKYGBgCgpOZXh0LCBjYWxjdWxhdGUgdGhlIG1ldHJpY3Mgb24gdGhpcyBtb2RlbCB1c2luZyBvdXIgaGVscGVyIGZ1bmN0aW9uLCBjYWxjX21ldHJpY3MoKS4gVGhlIE1BRSBvZiB0aGUgcHJlZGljdGlvbnMgaXMgYXBwcm94aW1hdGVseSB0aGUgc2FtZSBhcyB0aGUgbGluZWFyIG1vZGVsIGF0ICsvLTE5IEVucm9sbG1lbnRzIHBlciBkYXkuCmBgYHtyfQptb2RlbF8wMl9kZWNpc2lvbl90cmVlICU+JSAKICBjYWxjX21ldHJpY3ModGVzdF90YmwpICU+JSAKICBrbml0cjo6a2FibGUoKQpgYGAKCmBgYHtyLCB3YXJuaW5nPUZBTFNFfQptb2RlbF8wMl9kZWNpc2lvbl90cmVlICU+JSAKICBwbG90X3ByZWRpY3Rpb25zKHRlc3RfdGJsKSAlPiUgCiAgbGF5b3V0KHRpdGxlID0gIlByZWRpY3Rpb24gdnMgQWN0dWFsIEVucm9sbG1lbnRzOiBEZWNpc2lvbiBUcmVlIikKYGBgCgpGaW5hbGx5LCB1c2UgcnBhcnQucGxvdCgpIHRvIHZpc3VhbGl6ZSB0aGUgZGVjaXNpb24gdHJlZSBydWxlcy4gTm90ZSB0aGF0IHdlIG5lZWQgdG8gZXh0cmFjdCB0aGUgdW5kZXJseWluZyDigJxycGFydOKAnSBtb2RlbCBmcm9tIHRoZSBwYXJzbmlwIG1vZGVsIG9iamVjdCB1c2luZyB0aGUgbW9kZWxfMDJfZGVjaXNpb25fdHJlZSRmaXQKYGBge3J9Cm1vZGVsXzAyX2RlY2lzaW9uX3RyZWUkZml0ICU+JSAKICBycGFydC5wbG90KHJvdW5kaW50ID0gRkFMU0UsIAogICAgICAgICAgICAgY2V4ID0gMC44LCAKICAgICAgICAgICAgIGZhbGxlbi5sZWF2ZXMgPSBUUlVFLCAKICAgICAgICAgICAgIGV4dHJhID0gMTAxLCAKICAgICAgICAgICAgIG1haW4gPSAiTW9kZWwgMDI6IERlY2lzaW9uIFRyZWUiKQpgYGAKSW50ZXJwcmV0aW5nIHRoZSBkZWNpc2lvbiB0cmVlIGlzIHN0cmFpZ2h0Zm9yd2FyZDogRWFjaCBkZWNpc2lvbiBpcyBhIHJ1bGUsIGFuZCBZZXMgaXMgdG8gdGhlIGxlZnQsIE5vIGlzIHRvIHRoZSByaWdodC4gVGhlIHRvcCBmZWF0dXJlcyBhcmUgdGhlIG1vc3QgaW1wb3J0YW50IHRvIHRoZSBtb2RlbCAo4oCcUGFnZXZpZXdz4oCdIGFuZCDigJxDbGlja3PigJ0pLiBUaGUgZGVjaXNpb24gdHJlZSBzaG93cyB0aGF0IOKAnEV4cGVyaW1lbnTigJ0gaXMgaW52b2x2ZWQgaW4gdGhlIGRlY2lzaW9uIHJ1bGVzLiBUaGUgcnVsZXMgaW5kaWNhdGUgYSB3aGVuIEV4cGVyaW1lbnQgPj0gMC41LCB0aGVyZSBpcyBhIGRyb3AgaW4gZW5yb2xsbWVudHMuCgpLZXkgUG9pbnRzOgoKT3VyIG5ldyBtb2RlbCBoYXMgcm91Z2hseSB0aGUgc2FtZSBhY2N1cmFjeSB0byArLy0xOSBlbnJvbGxtZW50cyAoTUFFKSBhcyB0aGUgbGluZWFyIHJlZ3Jlc3Npb24gbW9kZWwuCgpFeHBlcmltZW50IHNob3dzIHVwIHRvd2FyZHMgdGhlIGJvdHRvbSBvZiB0aGUgdHJlZS4gVGhlIHJ1bGVzIGluZGljYXRlIGEgd2hlbiBFeHBlcmltZW50ID49IDAuNSwgdGhlcmUgaXMgYSBkcm9wIGluIGVucm9sbG1lbnRzLgoKIyMgWEdCb29zdApTZXZlcmFsIGtleSB0dW5pbmcgcGFyYW1ldGVycyBpbmNsdWRlOgotIG10cnk6IFRoZSBudW1iZXIgb2YgcHJlZGljdG9ycyB0aGF0IHdpbGwgYmUgcmFuZG9tbHkgc2FtcGxlZCBhdCBlYWNoIHNwbGl0IHdoZW4gY3JlYXRpbmcgdGhlIHRyZWUgbW9kZWxzLgotIHRyZWVzOiBUaGUgbnVtYmVyIG9mIHRyZWVzIGNvbnRhaW5lZCBpbiB0aGUgZW5zZW1ibGUuCi0gbWluX246IFRoZSBtaW5pbXVtIG51bWJlciBvZiBkYXRhIHBvaW50cyBpbiBhIG5vZGUgdGhhdCBhcmUgcmVxdWlyZWQgZm9yIHRoZSBub2RlIHRvIGJlIHNwbGl0IGZ1cnRoZXIuCi0gdHJlZV9kZXB0aDogVGhlIG1heGltdW0gZGVwdGggb2YgdGhlIHRyZWUgKGkuZS4gbnVtYmVyIG9mIHNwbGl0cykuCi0gbGVhcm5fcmF0ZTogVGhlIHJhdGUgYXQgd2hpY2ggdGhlIGJvb3N0aW5nIGFsZ29yaXRobSBhZGFwdHMgZnJvbSBpdGVyYXRpb24tdG8taXRlcmF0aW9uLgotIGxvc3NfcmVkdWN0aW9uOiBUaGUgcmVkdWN0aW9uIGluIHRoZSBsb3NzIGZ1bmN0aW9uIHJlcXVpcmVkIHRvIHNwbGl0IGZ1cnRoZXIuCi0gc2FtcGxlX3NpemU6IFRoZSBhbW91bnQgb2YgZGF0YSBleHBvc2VkIHRvIHRoZSBmaXR0aW5nIHJvdXRpbmUuCgpUaGUgcGFyYW1ldGVycyBzZWxlY3RlZCBmb3IgdGhlIG1vZGVsIHdlcmUgZGV0ZXJtaW5lZCB1c2luZyA1LWZvbGQgY3Jvc3MgdmFsaWRhdGlvbiB0byBwcmV2ZW50IG92ZXItZml0dGluZy4KCmBgYHtyfQpzZXQuc2VlZCgxMjMpCgptb2RlbF8wM194Z2Jvb3N0IDwtIGJvb3N0X3RyZWUoCiAgICBtb2RlID0gInJlZ3Jlc3Npb24iLCAKICAgIG10cnkgPSAxMDAsIAogICAgdHJlZXMgPSAxMDAwLCAKICAgIG1pbl9uID0gOCwgCiAgICB0cmVlX2RlcHRoID0gNiwgCiAgICBsZWFybl9yYXRlID0gMC4yLCAKICAgIGxvc3NfcmVkdWN0aW9uID0gMC4wMSwgCiAgICBzYW1wbGVfc2l6ZSA9IDEpICU+JSAKICBzZXRfZW5naW5lKCJ4Z2Jvb3N0IikgJT4lIAogIGZpdChFbnJvbGxtZW50cyB+IC4sIGRhdGEgPSB0cmFpbl90YmwgJT4lIHNlbGVjdCgtcm93X2lkKSkKYGBgCgpgYGB7cn0KbW9kZWxfMDNfeGdib29zdCAlPiUgCiAgY2FsY19tZXRyaWNzKHRlc3RfdGJsKSAlPiUgCiAga25pdHI6OmthYmxlKCkKYGBgCgpgYGB7ciB3YXJuaW5nPUZBTFNFfQptb2RlbF8wM194Z2Jvb3N0ICU+JSAKICBwbG90X3ByZWRpY3Rpb25zKHRlc3RfdGJsKSAlPiUgCiAgbGF5b3V0KHRpdGxlID0gIlByZWRpY3Rpb24gdnMgQWN0dWFsIEVucm9sbG1lbnRzOiBYR0Jvb3N0IikKYGBgCiMjIyBGZWF0dXJlIEltcG9ydGFuY2UKYGBge3J9CnhnYm9vc3RfZmVhdHVyZV9pbXBvcnRhbmNlX3RibCA8LSBtb2RlbF8wM194Z2Jvb3N0JGZpdCAlPiUgCiAgeGdiLmltcG9ydGFuY2UobW9kZWwgPSAuKSAlPiUgCiAgYXNfdGliYmxlKCkgJT4lIAogIG11dGF0ZShGZWF0dXJlID0gYXNfZmFjdG9yKEZlYXR1cmUpICU+JSBmY3RfcmV2KCkpCgp4Z2Jvb3N0X2ZlYXR1cmVfaW1wb3J0YW5jZV90YmwgJT4lIGtuaXRyOjprYWJsZSgpCmBgYAoKCmBgYHtyfQp4Z2Jvb3N0X2ZlYXR1cmVfaW1wb3J0YW5jZV90YmwgJT4lIAogIG11dGF0ZShMYWJlbCA9IHBhc3RlMChyb3VuZChHYWluKjEwMCwgMSkpLCAiJSIpICU+JSAKICBwbG90X2x5KHggPSB+R2FpbiwgeSA9IH5GZWF0dXJlLCB0eXBlID0gInNjYXR0ZXIiLCBtb2RlID0gIm1hcmtlcnMiLCBuYW1lID0gfkxhYmVsKSAlPiUgCiAgbGF5b3V0KHRpdGxlID0gIlhHQm9vc3QgRmVhdHVyZSBJbXBvcnRhbmNlIikKYGBgClRoZSBpbmZvcm1hdGlvbiBnYWluIGlzIDkzJSBmcm9tIFBhZ2V2aWV3cyBhbmQgQ2xpY2tzIGNvbWJpbmVkLiBFeHBlcmltZW50IGhhcyBhYm91dCBhIDclIGNvbnRyaWJ1dGlvbiB0byBpbmZvcm1hdGlvbiBnYWluLCBpbmRpY2F0aW5nIGl04oCZcyBzdGlsbCBwcmVkaWN0aXZlIChqdXN0IG5vdCBuZWFybHkgYXMgbXVjaCBhcyBQYWdldmlld3MpLiBUaGlzIHRlbGxzIGEgc3RvcnkgdGhhdCBpZiBFbnJvbGxtZW50cyBhcmUgY3JpdGljYWwsIFVkYWNpdHkgc2hvdWxkIGZvY3VzIG9uIGdldHRpbmcgUGFnZXZpZXdzLgoKS2V5IFBvaW50czoKLSBUaGUgWEdCb29zdCBtb2RlbCBlcnJvciBoYXMgZHJvcHBlZCB0byArLy0xMSBFbnJvbGxtZW50cy4KLSBUaGUgWEdCb29zdCBzaG93cyB0aGF0IEV4cGVyaW1lbnQgcHJvdmlkZXMgYW4gaW5mb3JtYXRpb24gZ2FpbiBvZiA3JQotIFRoZSBYR0Jvb3N0IG1vZGVsIHRlbGxzIGEgc3RvcnkgdGhhdCBVZGFjaXR5IHNob3VsZCBiZSBmb2N1c2luZyBvbiBQYWdlIFZpZXdzIGFuZCBzZWNvbmRhcmlseSBDbGlja3MgdG8gbWFpbnRhaW4gb3IgaW5jcmVhc2UgRW5yb2xsbWVudHMuIFRoZSBmZWF0dXJlcyBkcml2ZSB0aGUgc3lzdGVtLgoKIyMgQnVzaW5lc3MgQ29uY2x1c2lvbnMKVGhlcmUgYXJlIHNldmVyYWwga2V5IGJlbmVmaXRzIHRvIHBlcmZvcm1pbmcgQS9CIFRlc3RpbmcgdXNpbmcgTWFjaGluZSBMZWFybmluZy4gVGhlc2UgaW5jbHVkZToKClVuZGVyc3RhbmRpbmcgdGhlIENvbXBsZXggU3lzdGVtIC0gV2UgZGlzY292ZXJlZCB0aGF0IHRoZSBzeXN0ZW0gaXMgZHJpdmVuIGJ5IFBhZ2V2aWV3cyBhbmQgQ2xpY2tzLiBTdGF0aXN0aWNhbCBJbmZlcmVuY2Ugd291bGQgbm90IGhhdmUgaWRlbnRpZmllZCB0aGVzZSBkcml2ZXJzLiBNYWNoaW5lIExlYXJuaW5nIGRpZC4KClByb3ZpZGluZyBhIGRpcmVjdGlvbiBhbmQgbWFnbml0dWRlIG9mIHRoZSBleHBlcmltZW50IC0gV2Ugc2F3IHRoYXQgRXhwZXJpbWVudCA9IDEgZHJvcHMgZW5yb2xsbWVudHMgYnkgLTE3LjYgRW5yb2xsbWVudHMgUGVyIERheSBpbiB0aGUgTGluZWFyIFJlZ3Jlc3Npb24uIFdlIHNhdyBzaW1pbGFyIGRyb3BzIGluIHRoZSBEZWNpc2lvbiBUcmVlIHJ1bGVzLiBTdGF0aXN0aWNhbCBpbmZlcmVuY2Ugd291bGQgbm90IGhhdmUgaWRlbnRpZmllZCBtYWduaXR1ZGUgYW5kIGRpcmVjdGlvbi4gT25seSB3aGV0aGVyIG9yIG5vdCB0aGUgRXhwZXJpbWVudCBoYWQgYW4gZWZmZWN0LgoKV2hhdCBTaG91bGQgVWRhY2l0eSBEbz8KCklmIFVkYWNpdHkgd2FudHMgdG8gbWF4aW1pbWl6ZSBlbnJvbGxtZW50cywgaXQgc2hvdWxkIGZvY3VzIG9uIGluY3JlYXNpbmcgUGFnZSBWaWV3cyBmcm9tIHF1YWxpZmllZCBjYW5kaWRhdGVzLiBQYWdlIFZpZXdzIGlzIHRoZSBtb3N0IGltcG9ydGFudCBmZWF0dXJlIGluIDIgb2YgMyBtb2RlbHMuCgpJZiBVZGFjaXR5IHdhbnRzIGFsZXJ0IHBlb3BsZSBvZiB0aGUgdGltZSBjb21taXRtZW50LCB0aGUgYWRkaXRpb25hbCBwb3B1cCBmb3JtIGlzIGV4cGVjdGVkIHRvIGRlY3JlYXNlIHRoZSBudW1iZXIgb2YgZW5yb2xsbWVudHMuIFRoZSBuZWdhdGl2ZSBpbXBhY3QgY2FuIGJlIHNlZW4gaW4gdGhlIGRlY2lzaW9uIHRyZWUgKHdoZW4gRXhwZXJpbWVudCA8PSAwLjUsIEVucm9sbG1lbnRzIGdvIGRvd24pIGFuZCBpbiB0aGUgbGluZWFyIHJlZ3Jlc3Npb24gbW9kZWwgdGVybSAoLTE3LjYgRW5yb2xsbWVudHMgd2hlbiBFeHBlcmltZW50ID0gMSkuIElzIHRoaXMgT0s/IEl0IGRlcGVuZHMgb24gd2hhdCBVZGFjaXR54oCZcyBnb2FscyBhcmUuCgojIyBDcm9zcyBWYWxpZGF0aW9uIGFuZCBJbXByb3ZpbmcgTW9kZWxpbmcgUGVyZm9ybWFuY2UKVHdvIGltcG9ydGFudCBmdXJ0aGVyIGNvbnNpZGVyYXRpb25zIHdoZW4gaW1wbGVtZW50aW5nIGFuIEEvQiBUZXN0IHVzaW5nIE1hY2hpbmUgTGVhcm5pbmcgYXJlOgoxLiBIb3cgdG8gSW1wcm92ZSBNb2RlbGluZyBQZXJmb3JtYW5jZQoyLiBUaGUgbmVlZCBmb3IgQ3Jvc3MtVmFsaWRhdGlvbiBmb3IgVHVuaW5nIE1vZGVsIFBhcmFtZXRlcnMKCiMjIyBJbXByb3ZpbmcgTW9kZWxpbmcgUGVyZm9ybWFuY2UKLSBydW4gYW5hbHlzaXMgb24gdW5hZ2dyZWdhdGVkIGRhdGEgKGRhdGEgaW4gdGhpcyBleGVyY2lzZSB3YXMgYWdncmVnYXRlZCkKLSBydW4gYW5hbHlzaXMgb24gaW5kaXZpZHVhbCBjdXN0b21lciBkYXRhIHRvIGRldGVybWluZSBwcm9iYWJpbGl0eSBvbiBhbiBpbmRpdmlkdWFsIGN1c3RvbWVyIGVucm9sbGluZwotIGluY2x1ZGUgZ29vZCBmZWF0dXJlczsgY3VzdG9tZXItcmVsYXRlZCBmZWF0dXJlcyBub3QgaW5jbHVkZWQgaW4gdGhpcyBkYXRhIHNldAoKIyMjIENyb3NzLVZhbGlkYXRpb24gZm9yIFR1bmluZyBNb2RlbHMKLSBJbiBwcmFjdGljZSwgd2UgbmVlZCB0byBwZXJmb3JtIGNyb3NzLXZhbGlkYXRpb24gdG8gcHJldmVudCB0aGUgbW9kZWxzIGZyb20gYmVpbmcgdHVuZWQgdG8gdGhlIHRlc3QgZGF0YSBzZXQuCgojIyBVc2luZyBjYXJldAoKYGBge3J9CmBgYAoKCmBgYHtyfQpgYGAKCgpgYGB7cn0KYGBgCgo=